import requests

from lxml import etree
import json

class BtcSpider(object):
    url = "http://8btc.com/"

    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
    }

    result_list = []

    # 请求数据的方法
    def get_data(self,url):
        data = requests.get(url, headers=self.headers).content.decode('gbk')
        return data

    # 解析数据的方法
    def parse(self, data,url):
        parse_data = etree.HTML(data)

        if data is not None:
            div_list = parse_data.xpath('//form[@id="moderate"]/div[@class="discuz-list clearfix"]/div[2]')

            for div in div_list:
                dict = {}
                dict['title'] = div.xpath('.//a[@class="s xst"]/text()')[0]
                dict['url'] = div.xpath('.//a[@class="s xst"]/@href')[0]

                self.result_list.append(dict)
        else:
            print("没有数据的页面:")
            print(url)


    # 保存数据
    def save_data(self):
        with open('02btx.json', 'w') as f:
            # 将list 转换成 str
            btx_str = json.dumps(self.result_list)
            f.write(btx_str)

    # 运行的方法
    def run(self):

        # 翻页请求
        for i in range(1,11):
            url = "http://8btc.com/forum-61-" + str(i) + ".html"

            print(url)
            # 1.请求首页
            home_data = self.get_data(url)

            # 2.解析数据
            self.parse(home_data,url)
        # 3.保存
        self.save_data()


spider = BtcSpider()
spider.run()







"""
     # 解析所有的 title -->url
        title_list = parse_data.xpath('//a[@class="s xst"]/text()')
        title_url_list = parse_data.xpath('//a[@class="s xst"]/@href')
       
        list = ["a","b","c"]

        result_list = []

        for index,a in enumerate(title_list):
            dict = {}
            dict['title'] = a
            dict['url'] = title_url_list[index]

            result_list.append(dict)

        print(result_list)


"""